import scrapy
from urllib.parse import urljoin
from copy import deepcopy
from scrapy_redis.spiders import RedisSpider


class DangdangSpider(RedisSpider):
    name = 'dangdang'
    allowed_domains = ['dangdang.com']
    # start_urls = ['https://book.dangdang.com/']
    redis_key = "dang_dang"

    def parse(self, response, *args):
        # 设置大分类
        div_list = response.xpath('//div[@id="bd_auto"]//div[@class="con flq_body"]/div')
        for div in div_list:
            item = dict()
            big_cate = div.xpath('./dl/dt//text()').extract()  # 大分类名
            # 大分类名中有多个分类，所以我们要处理一下,判断一下是否为空
            item['big_cate'] = [i.strip().replace('xa0', '').replace("', '", '') for i in big_cate if
                                len(i.strip()) > 0]
            # 设置中间分类
            dl_list = div.xpath('.//div[@class="col eject_left"]/dl')
            for dl in dl_list:
                item['mid_cate'] = dl.xpath('./dt/text()').extract_first().strip()
                if not item['mid_cate']:
                    item['mid_cate'] = dl.xpath('./dt/a/text()').extract_first().strip()
                a_list = dl.xpath('./dd/a')
                for a in a_list:
                    # 获取小分类名称
                    item["small_cate"] = a.xpath('./@title').extract_first()
                    # 获取url地址,注意这里要做一个转换，不转换报错
                    small_url = a.xpath('./@href').extract_first()
                    item['small_url'] = small_url
                    if item['small_url'] is not None:
                        yield scrapy.Request(
                            small_url,
                            callback=self.detail_book_list,
                            meta={'item': deepcopy(item)},
                        )

    def detail_book_list(self, response):
        item = deepcopy(response.meta['item'])
        li_list = response.xpath('//ul[@class="bigimg"]/li')
        for li in li_list:
            """书籍列表页中获取书籍图片、书名、书籍介绍、当前价 格、基本定价、书籍折扣"""
            item['book_pic'] = li.xpath('./a[@class="pic"]/img/@src').extract_first()
            # 判断pic是否是无图
            if item['book_pic'] == 'images/model/guan/url_none.png':
                item['book_pic'] = li.xpath('./a[@class="pic"]/img/@data-original').extract_first()
            item['book_pic'] = urljoin('http:', item['book_pic'])
            item['book_name'] = li.xpath('./p[@class="name"]/a/@title').extract_first()
            item['book_detail'] = li.xpath('./p[@class="detail"]/text()').extract_first()
            item['book_price_now'] = li.xpath(
                './p[@class="price"]/span[@class="search_now_price"]/text()').extract_first()
            item['book_price_base'] = li.xpath(
                './p[@class="price"]/span[@class="search_pre_price"]/text()').extract_first()
            item['book_price_discount'] = li.xpath(
                './p[@class="price"]/span[@class="search_discount"]/text()').extract_first()
            # 有些打折，有些没有打折
            if item['book_price_discount'] is not None:
                item['book_price_discount'] = item['book_price_discount'].strip()

            item['book_evaluate_num'] = li.xpath('./p[@class="search_star_line"]/a/text()').extract_first()
            item['book_author'] = li.xpath('./p[@class="search_book_author"]/span[1]/a/text()').extract()
            item['book_press'] = li.xpath('./p[@class="search_book_author"]/span[3]/a/text()').extract_first()
            item['book_publish_data'] = li.xpath(
                './p[@class="search_book_author"]/span[2]/text()').extract_first()
            if item['book_publish_data']:
                item['book_publish_data'] = item['book_publish_data'].replace(' /', '')
            print(item)
        """构建下一页内容"""
        next_url= response.xpath('//ul[@name="Fy"]/li[@class="next"]/a/@href').extract_first()
        if next_url is not None:
            next_url = urljoin(response.url,next_url)
            yield scrapy.Request(
                next_url,
                callback=self.detail_book_list,
                meta={'item':item}
            )

